/* 
 * Copyright 2011 Carnegie Mellon University
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package edu.cmu.lti.dimple.nlp;

import java.util.Arrays;
import java.util.List;

/**
 * English specific NLP implementation. 
 * 
 * @author Hideki Shima
 *
 */
public class EnglishNLP extends AbstractNLPTool {

  static {
    SnowballStemmer.create();
  }
  
  public final static String GENERAL_TERMS = "out|in|inside|into|by|of|at|for|under|until|from" +
      "|on|with|within|without|via|versus|about|above|across|after|against|along|among|around" +
      "|before|behind|below|beneath|beside|besides|between|beyond|except|following|to|up|upon|down" +
      "|or|outside|over|through|toward|towards|and|but|no|not|all|s|The|the|a|an|as|more|than|if|some" +
      "|can|will|other|another|well|then|since" +
      "|my|me|your|you|his|him|he|she|us|it|her|hers|they|them|theirs|this|that|there" +
      "|who|what|which|how|where|when|while" +
      "|is|are|be|am|been|being|were|was|has|have|had|make|made|do|did|done";
  
  @Override
  public String stem(String word) {
    return SnowballStemmer.stem( word );
  }

  @Override
  public List<String> extractContentWords( String text ) {
    text = text.replaceAll("<.+?>", ""); // remove variables
    text = text.replaceAll("[^a-zA-Z0-9]", " "); // remove non-ascii
    text = text.replaceAll("\\b(?i)("+GENERAL_TERMS+")\\b", " ");
    text = text.replaceAll("[ ]+", " ");
    text = text.trim(); // remove trailing spaces
    return Arrays.asList(text.split(" "));
  }
  
}
